{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "couldn't import doomish\n",
      "Couldn't import doom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:187: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: The result returned by `env.reset()` was not a tuple of the form `(obs, info)`, where `obs` is a observation and `info` is a dictionary containing additional information. Actual type: `<class 'numpy.ndarray'>`\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([24.,  0.,  7., 17., 49., 31., 40.])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "import gym_pygame\n",
    "\n",
    "#!pip install git+https://github.com/ntasfi/PyGame-Learning-Environment.git\n",
    "#!pip install git+https://github.com/qlan3/gym-games.git\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pixelcopter-PLE-v0')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box(-inf, inf, (7,), float32)\n",
      "env.action_space= Discrete(2)\n",
      "state= [24.  0.  7. 17. 46. 23. 32.]\n",
      "action= 0\n",
      "next_state= [23.4456 -0.5544  6.4456 17.5544 45.36   23.     32.    ]\n",
      "reward= 0.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
      "  logger.deprecation(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n",
      "  logger.warn(f\"{pre} is not within the observation space.\")\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:272: UserWarning: \u001b[33mWARN: No render modes was declared in the environment (env.metadata['render_modes'] is None or not defined), you may have trouble when calling `.render()`.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAEVCAYAAAAl2crhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAATkElEQVR4nO3df2zV1f3H8Vdrey9i29sW9V4bel0TjY0jdLEC3rhkGW0ghBig/cMl/tGImQFuCdAlG00m/rPkNpoo4tCZueH2x9alJmBw0dEUuWRZYfRCIwJ2LiHjJu29nX/cH1Z62/Se7x9+vfNKgd5yyv0Un4/kndjzOfdz358mfXk493PvLTHGGAGARaXFbgDAnYdgAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWBd2UKd+ODBg3r55ZcVi8XU1NSk119/XatXr77p47LZrEZHR1VZWamSkpKFag9AgYwxSqfTqqurU2npTdYkZgH09vYal8tlfv/735sLFy6Yn/70p6a6utrE4/GbPjYajRpJFEU5tKLR6E3/jkuMsf8mxDVr1mjVqlX69a9/LemrVUh9fb127typvXv33vCxyWRS1dXV2nFhndyV5bZbAzBPmfS03vj+MSUSCXk8nhvOtf5PoampKUUiEXV3d+fGSktL1draqsHBwWubzWSUyWRyP6fTaUmSu7Jc7iqCBXCauWxRWN+8/fzzzzUzMyOv15s37vV6FYvFrpkfCoXk8XhyVV9fb7slALdZ0V8V6u7uVjKZzFU0Gi12SwBukfV/Ct1777266667FI/H88bj8bh8Pt81891ut9xut+02ABSR9RWLy+VSc3OzBgYGcmPZbFYDAwMKBAK2nw6AAy3IfSxdXV3q6OjQ448/rtWrV2v//v2amJjQs88+uxBPB8BhFiRYnn76af33v//Vvn37FIvF9IMf/EAffvjhNRu6AO5MC3bnbWdnpzo7Oxfq9AAcrOivCgG48xAsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYt2CfeQvg9uupPlLsFiSxYgGwAAgWANYRLACsI1gAWMfmLVBkTtlwtYkVCwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHXcIAcU6E68oc02ViwArCNYAFhHsACwruBgOXnypJ566inV1dWppKRER44cyTtujNG+ffv0wAMP6O6771Zra6s+++wzW/0CWAQK3rydmJhQU1OTtm7dqra2tmuOv/TSSzpw4ID+8Ic/qKGhQS+88ILWr1+vixcvasmSJVaaBgrFhuvtVXCwbNiwQRs2bJj1mDFG+/fv1y9/+Utt2rRJkvTHP/5RXq9XR44c0U9+8pNb6xbAomB1j+Xy5cuKxWJqbW3NjXk8Hq1Zs0aDg4OzPiaTySiVSuUVgMXNarDEYjFJktfrzRv3er25Y98WCoXk8XhyVV9fb7MlAEVQ9FeFuru7lUwmcxWNRovdEoBbZDVYfD6fJCkej+eNx+Px3LFvc7vdqqqqyisAi5vVYGloaJDP59PAwEBuLJVK6fTp0woEAjafCoCDFfyq0BdffKF///vfuZ8vX76s4eFh1dbWyu/3a/fu3frVr36lhx9+OPdyc11dnTZv3myzbwAOVnCwDA0N6cc//nHu566uLklSR0eH3nnnHf385z/XxMSEnn/+eSUSCf3whz/Uhx9+yD0swHdIiTHGFLuJb0qlUvJ4PNpzZaPcVeXFbgd3CG6QsyeZTN50L7TorwoBuPMQLACsI1gAWEewALCOYAFgHZ95C6t49QUSKxYAC4BgAWAdwQLAOsfusbzq/2vBj9mb2Gy/EQAFY8UCwDqCBYB1BAsA6wgWANY5dvN2PmzfnMVmMDA/rFgAWEewALCOYAFgHcECwLo7avPWNpubwWwE47uEFQsA6wgWANYRLACsI1gAWOfYLywD4Ex8YRmAoiBYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYF1BwRIKhbRq1SpVVlbq/vvv1+bNmzUyMpI3Z3JyUsFgUMuWLVNFRYXa29sVj8etNg3A2QoKlnA4rGAwqFOnTqm/v1/T09Nat26dJiYmcnP27Nmjo0ePqq+vT+FwWKOjo2pra7PeOAAHM7dgfHzcSDLhcNgYY0wikTDl5eWmr68vN+fSpUtGkhkcHJzTOZPJpJFEUZRDK5lM3vTv+Jb2WJLJpCSptrZWkhSJRDQ9Pa3W1tbcnMbGRvn9fg0ODs56jkwmo1QqlVcAFrd5B0s2m9Xu3bv15JNPasWKFZKkWCwml8ul6urqvLler1exWGzW84RCIXk8nlzV19fPtyUADjHvYAkGg/rkk0/U29t7Sw10d3crmUzmKhqN3tL5ABTfvL5XqLOzU++//75Onjyp5cuX58Z9Pp+mpqaUSCTyVi3xeFw+n2/Wc7ndbrnd7vm0AcCpCtmszWazJhgMmrq6OvOvf/3rmuNfb96+++67ubFPP/3USGzeUtSdUnPZvC0oWLZv3248Ho85ceKEGRsby9WXX36Zm7Nt2zbj9/vN8ePHzdDQkAkEAiYQCMz5OQgWinJ2WQ+W6z3RoUOHcnOuXr1qduzYYWpqaszSpUvNli1bzNjYGMFCUXdIzSVY+PoPAAWZy9d/8KXwi9Bc/19QUlKywJ0As+NNiACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANZxg9wixI1vcDpWLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCuoGB58803tXLlSlVVVamqqkqBQEAffPBB7vjk5KSCwaCWLVumiooKtbe3Kx6PW28agLMVFCzLly9XT0+PIpGIhoaGtHbtWm3atEkXLlyQJO3Zs0dHjx5VX1+fwuGwRkdH1dbWtiCNA3Awc4tqamrM22+/bRKJhCkvLzd9fX25Y5cuXTKSzODg4JzPl0wmjSSKohxayWTypn/H895jmZmZUW9vryYmJhQIBBSJRDQ9Pa3W1tbcnMbGRvn9fg0ODl73PJlMRqlUKq8ALG4FB8v58+dVUVEht9utbdu26fDhw3r00UcVi8XkcrlUXV2dN9/r9SoWi133fKFQSB6PJ1f19fUFXwQAZyk4WB555BENDw/r9OnT2r59uzo6OnTx4sV5N9Dd3a1kMpmraDQ673MBcIayQh/gcrn00EMPSZKam5t15swZvfbaa3r66ac1NTWlRCKRt2qJx+Py+XzXPZ/b7Zbb7S68cwCOdcv3sWSzWWUyGTU3N6u8vFwDAwO5YyMjI7py5YoCgcCtPg2ARaSgFUt3d7c2bNggv9+vdDqtP/3pTzpx4oT+9re/yePx6LnnnlNXV5dqa2tVVVWlnTt3KhAI6Iknnlio/gE4USEvLW/dutU8+OCDxuVymfvuu8+0tLSYY8eO5Y5fvXrV7Nixw9TU1JilS5eaLVu2mLGxsUKegpebKcrhNZeXm0uMMUYOkkql5PF4it0GgOtIJpOqqqq64ZyCN28BONfexOYFO3cmNa1X/X+d01zehAjAOoIFgHUECwDrCBYA1rF5CxTZQm64FgsrFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOu4QQ4o0J14Q5ttrFgAWEewALCOYAFgHcECwDo2b/GdwIbr7cWKBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOj02AVXw8ASRWLAAWAMECwDqCBYB1t7TH0tPTo+7ubu3atUv79++XJE1OTupnP/uZent7lclktH79er3xxhvyer0FnXvPlY1yV5UX1k/1kYLmA1gY816xnDlzRm+99ZZWrlyZN75nzx4dPXpUfX19CofDGh0dVVtb2y03CmDxmFewfPHFF3rmmWf029/+VjU1NbnxZDKp3/3ud3rllVe0du1aNTc369ChQ/rHP/6hU6dOzXquTCajVCqVVwAWt3kFSzAY1MaNG9Xa2po3HolEND09nTfe2Ngov9+vwcHBWc8VCoXk8XhyVV9fP5+WADhIwcHS29urs2fPKhQKXXMsFovJ5XKpuro6b9zr9SoWi816vu7ubiWTyVxFo9FCWwLgMAVt3kajUe3atUv9/f1asmSJlQbcbrfcbreVc9m+OYvNYGB+ClqxRCIRjY+P67HHHlNZWZnKysoUDod14MABlZWVyev1ampqSolEIu9x8XhcPp/PZt8AHKygFUtLS4vOnz+fN/bss8+qsbFRv/jFL1RfX6/y8nINDAyovb1dkjQyMqIrV64oEAjY6xqAoxUULJWVlVqxYkXe2D333KNly5blxp977jl1dXWptrZWVVVV2rlzpwKBgJ544gl7XQNwNOtvQnz11VdVWlqq9vb2vBvkAHx3lBhjTLGb+KZUKiWPxzOvO2+d7LuyEcy7m+9cmdS0XvX/VclkUlVVVTecy3uFAFhHsACwjmABYB3BAsA6PpryNmFTE98lrFgAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6xz39R9ff5V0Jj1d5E4AfNPXf5Nz+bp3xwVLOp2WJL3x/WNF7gTAbNLptDwezw3nlJi5xM9tlM1mNTo6qsrKSqXTadXX1ysajd702+2dKJVK0X8R0b9dxhil02nV1dWptPTGuyiOW7GUlpZq+fLlkqSSkhJJUlVVlSN+sfNF/8VF//bcbKXyNTZvAVhHsACwztHB4na79eKLL8rtdhe7lXmh/+Ki/+Jx3OYtgMXP0SsWAIsTwQLAOoIFgHUECwDrCBYA1jk2WA4ePKjvfe97WrJkidasWaN//vOfxW7puk6ePKmnnnpKdXV1Kikp0ZEjR/KOG2O0b98+PfDAA7r77rvV2tqqzz77rDjNfksoFNKqVatUWVmp+++/X5s3b9bIyEjenMnJSQWDQS1btkwVFRVqb29XPB4vUsf53nzzTa1cuTJ3d2ogENAHH3yQO+7k3mfT09OjkpIS7d69Oze22K5Bcmiw/OUvf1FXV5defPFFnT17Vk1NTVq/fr3Gx8eL3dqsJiYm1NTUpIMHD856/KWXXtKBAwf0m9/8RqdPn9Y999yj9evXa3Jy8jZ3eq1wOKxgMKhTp06pv79f09PTWrdunSYmJnJz9uzZo6NHj6qvr0/hcFijo6Nqa2srYtf/s3z5cvX09CgSiWhoaEhr167Vpk2bdOHCBUnO7v3bzpw5o7feeksrV67MG19M15BjHGj16tUmGAzmfp6ZmTF1dXUmFAoVsau5kWQOHz6c+zmbzRqfz2defvnl3FgikTBut9v8+c9/LkKHNzY+Pm4kmXA4bIz5qtfy8nLT19eXm3Pp0iUjyQwODharzRuqqakxb7/99qLqPZ1Om4cfftj09/ebH/3oR2bXrl3GmMX5+zfGGMetWKamphSJRNTa2pobKy0tVWtrqwYHB4vY2fxcvnxZsVgs73o8Ho/WrFnjyOtJJpOSpNraWklSJBLR9PR0Xv+NjY3y+/2O639mZka9vb2amJhQIBBYVL0Hg0Ft3Lgxr1dpcf3+v8lx727+/PPPNTMzI6/Xmzfu9Xr16aefFqmr+YvFYpI06/V8fcwpstmsdu/erSeffFIrVqyQ9FX/LpdL1dXVeXOd1P/58+cVCAQ0OTmpiooKHT58WI8++qiGh4cd37sk9fb26uzZszpz5sw1xxbD7382jgsWFE8wGNQnn3yiv//978VupSCPPPKIhoeHlUwm9e6776qjo0PhcLjYbc1JNBrVrl271N/fryVLlhS7HWsc90+he++9V3fdddc1u97xeFw+n69IXc3f1z07/Xo6Ozv1/vvv66OPPsp9Ho70Vf9TU1NKJBJ5853Uv8vl0kMPPaTm5maFQiE1NTXptddeWxS9RyIRjY+P67HHHlNZWZnKysoUDod14MABlZWVyev1Ov4aZuO4YHG5XGpubtbAwEBuLJvNamBgQIFAoIidzU9DQ4N8Pl/e9aRSKZ0+fdoR12OMUWdnpw4fPqzjx4+roaEh73hzc7PKy8vz+h8ZGdGVK1cc0f9sstmsMpnMoui9paVF58+f1/DwcK4ef/xxPfPMM7n/dvo1zKrYu8ez6e3tNW6327zzzjvm4sWL5vnnnzfV1dUmFosVu7VZpdNpc+7cOXPu3Dkjybzyyivm3Llz5j//+Y8xxpienh5TXV1t3nvvPfPxxx+bTZs2mYaGBnP16tUid27M9u3bjcfjMSdOnDBjY2O5+vLLL3Nztm3bZvx+vzl+/LgZGhoygUDABAKBInb9P3v37jXhcNhcvnzZfPzxx2bv3r2mpKTEHDt2zBjj7N6v55uvChmzOK/BkcFijDGvv/668fv9xuVymdWrV5tTp04Vu6Xr+uijj4yka6qjo8MY89VLzi+88ILxer3G7XablpYWMzIyUtym/99sfUsyhw4dys25evWq2bFjh6mpqTFLly41W7ZsMWNjY8Vr+hu2bt1qHnzwQeNyucx9991nWlpacqFijLN7v55vB8tivAY+jwWAdY7bYwGw+BEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABY939iNQecLNFczgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render(mode='rgb_array'))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7f603fe29280>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=1024,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-4.7, 0.45825756949558405)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b601c43c237f44869c96632ca93b5c4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">WARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to </span>\n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">rewrite the environment with new step API. </span>\n",
       "  logger.deprecation(\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \n",
       "\u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to \u001b[0m\n",
       "\u001b[33mrewrite the environment with new step API. \u001b[0m\n",
       "  logger.deprecation(\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: <span style=\"color: #808000; text-decoration-color: #808000\">WARN: The </span>\n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64</span>\n",
       "  logger.warn(\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The \u001b[0m\n",
       "\u001b[33mobs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
       "  logger.warn(\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: <span style=\"color: #808000; text-decoration-color: #808000\">WARN: The </span>\n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">obs returned by the `step()` method is not within the observation space.</span>\n",
       "  logger.warn(f\"{pre} is not within the observation space.\")\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The \u001b[0m\n",
       "\u001b[33mobs returned by the `step()` method is not within the observation space.\u001b[0m\n",
       "  logger.warn(f\"{pre} is not within the observation space.\")\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:187: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: The result returned by `env.reset()` was not a tuple of the form `(obs, info)`, where `obs` is a observation and `info` is a dictionary containing additional information. Actual type: `<class 'numpy.ndarray'>`\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/2.PPO.Pixelcopter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25.7, 20.25857842988989)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/2.PPO.Pixelcopter')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAEVCAYAAAAl2crhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAATuUlEQVR4nO3df2jV1/3H8VfS5F7bJrkxtr3XYLIFWpp1kpSmai8djGnWIFK0SaGDwkJbVtQbUTPYDKz2n8ENFaq1067sh93+6FJS0GFHf4RYr4xFZ66GWq1ZBzIvJPdm/SP33qbmJuSe7x/99q63Rs1NTno/V58POOA9n3PPfX8Cvjg59+TeImOMEQBYVJzvAgDcfAgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANaVLNbEBw4c0J49exSNRtXY2KhXX31Vq1evvuHz0um0RkZGVF5erqKiosUqD0COjDFKJpOqrq5WcfEN1iRmEfT09BiXy2X++Mc/mvPnz5uf/exnprKy0sRisRs+NxKJGEk0Gs2hLRKJ3PD/cZEx9v8Icc2aNVq1apV+85vfSPpyFVJTU6Nt27Zp165d131uPB5XZWWltp5/TO7yUtulAZinVHJaB7//gcbHx+XxeK471vqvQlNTUwqHw+rq6sr0FRcXq7m5WQMDA1cXm0oplUplHieTSUmSu7xU7gqCBXCauWxRWN+8/eyzzzQzMyOv15vV7/V6FY1GrxofDAbl8XgyraamxnZJAL5leX9XqKurS/F4PNMikUi+SwKwQNZ/Fbrrrrt02223KRaLZfXHYjH5fL6rxrvdbrndbttlAMgj6ysWl8ulpqYm9ff3Z/rS6bT6+/vl9/ttvxwAB1qUcyydnZ1qb2/Xww8/rNWrV2vfvn2amJjQM888sxgvB8BhFiVYnnrqKf33v//V7t27FY1G9eCDD+q99967akMXwM1p0U7ednR0qKOjY7GmB+BgeX9XCMDNh2ABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwriTfBVzL3tq/ZT3eNb4pP4UAyBkrFgDWESwArCNYAFiXc7CcOHFCjz/+uKqrq1VUVKQjR45kXTfGaPfu3Vq+fLluv/12NTc369NPP7VVL4ACkPPm7cTEhBobG/Xss8+qtbX1qusvvfSS9u/frz/96U+qq6vTCy+8oJaWFl24cEFLliyxUjSAhemuPLKo8+ccLOvXr9f69etnvWaM0b59+/SrX/1KGzdulCT9+c9/ltfr1ZEjR/STn/xkYdUCKAhW91guXbqkaDSq5ubmTJ/H49GaNWs0MDAw63NSqZQSiURWA1DYrAZLNBqVJHm93qx+r9ebufZNwWBQHo8n02pqamyWBCAP8v6uUFdXl+LxeKZFIpF8lwRggayevPX5fJKkWCym5cuXZ/pjsZgefPDBWZ/jdrvldrtvOLfNzSZO8QKLy+qKpa6uTj6fT/39/Zm+RCKhU6dOye/323wpAA6W84rl888/17///e/M40uXLmloaEhVVVWqra3Vjh079Otf/1r33Xdf5u3m6upqbdq0yWbdABws52AZHBzUj370o8zjzs5OSVJ7e7veeOMN/eIXv9DExISef/55jY+P6wc/+IHee+89zrAAt5AiY4zJdxFfl0gk5PF4FvU12GPBrW4he5bxeFwVFRXXHePYj00A8D+LfVLWtry/3Qzg5kOwALCOYAFgHcECwLpbcvOWU7zA4mLFAsA6ggWAdQQLAOtuyZO3Tsaezc2j0A61zdVcTt6yYgFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHSdvb2Kc4s3dzXpa1iZO3gLIC4IFgHUECwDrCBYA1t2SH01Z6Oa6315UVLTIlQCzY8UCwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHQfkChAH3+B0rFgAWEewALCOYAFgXU7BEgwGtWrVKpWXl+uee+7Rpk2bNDw8nDVmcnJSgUBAy5YtU1lZmdra2hSLxawWDcDZcgqWUCikQCCgkydPqq+vT9PT03rsscc0MTGRGbNz504dPXpUvb29CoVCGhkZUWtrq/XCATiYWYCxsTEjyYRCIWOMMePj46a0tNT09vZmxnzyySdGkhkYGJjTnPF43Eii0WgObfF4/Ib/jxe0xxKPxyVJVVVVkqRwOKzp6Wk1NzdnxtTX16u2tlYDAwOzzpFKpZRIJLIagMI272BJp9PasWOHHn30Ua1cuVKSFI1G5XK5VFlZmTXW6/UqGo3OOk8wGJTH48m0mpqa+ZYEwCHmHSyBQEAff/yxenp6FlRAV1eX4vF4pkUikQXNByD/5nXytqOjQ++8845OnDihFStWZPp9Pp+mpqY0Pj6etWqJxWLy+XyzzuV2u+V2u+dTBgCnymWzNp1Om0AgYKqrq82//vWvq65/tXn79ttvZ/ouXrxoJDZvabSbpc1l8zanYNmyZYvxeDzm+PHjZnR0NNO++OKLzJjNmzeb2tpac+zYMTM4OGj8fr/x+/1zfg2ChUZzdrMeLNd6oUOHDmXGXLlyxWzdutUsXbrU3HHHHeaJJ54wo6OjBAuNdpO0uQQL390MICd8dzOAvCBYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWzetL4XFz2zW+ydpc3ZVHrM2FwsGKBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsyylYXnvtNTU0NKiiokIVFRXy+/169913M9cnJycVCAS0bNkylZWVqa2tTbFYzHrRAJwtp2BZsWKFuru7FQ6HNTg4qLVr12rjxo06f/68JGnnzp06evSoent7FQqFNDIyotbW1kUpHIBzFRljzEImqKqq0p49e/Tkk0/q7rvv1ptvvqknn3xSknTx4kV973vf08DAgB555JE5zZdIJOTxeBZSEhaIj03A9cTjcVVUVFx3zLz3WGZmZtTT06OJiQn5/X6Fw2FNT0+rubk5M6a+vl61tbUaGBi45jypVEqJRCKrAShsOQfLuXPnVFZWJrfbrc2bN+vw4cN64IEHFI1G5XK5VFlZmTXe6/UqGo1ec75gMCiPx5NpNTU1Od8EAGfJOVjuv/9+DQ0N6dSpU9qyZYva29t14cKFeRfQ1dWleDyeaZFIZN5zAXCGnD+a0uVy6d5775UkNTU16fTp03rllVf01FNPaWpqSuPj41mrllgsJp/Pd8353G633G537pUDcKwFn2NJp9NKpVJqampSaWmp+vv7M9eGh4d1+fJl+f3+hb4MgAKS04qlq6tL69evV21trZLJpN58800dP35c77//vjwej5577jl1dnaqqqpKFRUV2rZtm/x+/5zfEQJwc8gpWMbGxvTTn/5Uo6Oj8ng8amho0Pvvv68f//jHkqS9e/equLhYbW1tSqVSamlp0cGDBxelcADOteBzLLZxjiX/OMeC61nUcywAcC0ECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsC7nj6b8tuy8vEHuitLrjuFP8gFnYsUCwDqCBYB1BAsA6wgWANY5dvMW+cOm+M1vPp9rnEpMa2/t3+Y0lhULAOsIFgDWESwArCNYAFhX0Ju38/1iLTYngcXFigWAdQQLAOsIFgDWESwArCvozVvgVjHfNyryhRULAOsIFgDWESwArCNYAFh3S27e2twI4xQvcDVWLACsI1gAWEewALBuQXss3d3d6urq0vbt27Vv3z5J0uTkpH7+85+rp6dHqVRKLS0tOnjwoLxer416Hcf2wSX2bG4ehXaozaZ5r1hOnz6t119/XQ0NDVn9O3fu1NGjR9Xb26tQKKSRkRG1trYuuFAAhWNewfL555/r6aef1u9+9zstXbo00x+Px/WHP/xBL7/8stauXaumpiYdOnRI//jHP3Ty5MlZ50qlUkokElkNQGGbV7AEAgFt2LBBzc3NWf3hcFjT09NZ/fX19aqtrdXAwMCscwWDQXk8nkyrqamZT0kAHCTnYOnp6dGZM2cUDAavuhaNRuVyuVRZWZnV7/V6FY1GZ52vq6tL8Xg80yKRSK4lAXCYnDZvI5GItm/frr6+Pi1ZssRKAW63W26328pcAJwhpxVLOBzW2NiYHnroIZWUlKikpEShUEj79+9XSUmJvF6vpqamND4+nvW8WCwmn89ns24ADpbTimXdunU6d+5cVt8zzzyj+vp6/fKXv1RNTY1KS0vV39+vtrY2SdLw8LAuX74sv99vr2oAjpZTsJSXl2vlypVZfXfeeaeWLVuW6X/uuefU2dmpqqoqVVRUaNu2bfL7/XrkkUfsVQ3A0az/EeLevXtVXFystra2rANyAG4dRcYYk+8ivi6RSMjj8Wjn5Q1yV5Tmu5yCxine3N3Kp2Vv5KsvhY/H46qoqLjuWP5WCIB1BAsA6wgWANYRLACsuyU/mvJWwUYk8oUVCwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWCd477+46uvkk4lp/NcCYCv++r/5Fy+7t1xwZJMJiVJB7//QZ4rATCbZDIpj8dz3TFFZi7x8y1Kp9MaGRlReXm5ksmkampqFIlEbvjt9k6USCSoP4+o3y5jjJLJpKqrq1VcfP1dFMetWIqLi7VixQpJUlFRkSSpoqLCET/Y+aL+/KJ+e260UvkKm7cArCNYAFjn6GBxu9168cUX5Xa7813KvFB/flF//jhu8xZA4XP0igVAYSJYAFhHsACwjmABYB3BAsA6xwbLgQMH9N3vfldLlizRmjVr9M9//jPfJV3TiRMn9Pjjj6u6ulpFRUU6cuRI1nVjjHbv3q3ly5fr9ttvV3Nzsz799NP8FPsNwWBQq1atUnl5ue655x5t2rRJw8PDWWMmJycVCAS0bNkylZWVqa2tTbFYLE8VZ3vttdfU0NCQOZ3q9/v17rvvZq47ufbZdHd3q6ioSDt27Mj0Fdo9SA4NlrfeekudnZ168cUXdebMGTU2NqqlpUVjY2P5Lm1WExMTamxs1IEDB2a9/tJLL2n//v367W9/q1OnTunOO+9US0uLJicnv+VKrxYKhRQIBHTy5En19fVpenpajz32mCYmJjJjdu7cqaNHj6q3t1ehUEgjIyNqbW3NY9X/s2LFCnV3dyscDmtwcFBr167Vxo0bdf78eUnOrv2bTp8+rddff10NDQ1Z/YV0DxnGgVavXm0CgUDm8czMjKmurjbBYDCPVc2NJHP48OHM43Q6bXw+n9mzZ0+mb3x83LjdbvOXv/wlDxVe39jYmJFkQqGQMebLWktLS01vb29mzCeffGIkmYGBgXyVeV1Lly41v//97wuq9mQyae677z7T19dnfvjDH5rt27cbYwrz52+MMY5bsUxNTSkcDqu5uTnTV1xcrObmZg0MDOSxsvm5dOmSotFo1v14PB6tWbPGkfcTj8clSVVVVZKkcDis6enprPrr6+tVW1vruPpnZmbU09OjiYkJ+f3+gqo9EAhow4YNWbVKhfXz/zrH/XXzZ599ppmZGXm93qx+r9erixcv5qmq+YtGo5I06/18dc0p0um0duzYoUcffVQrV66U9GX9LpdLlZWVWWOdVP+5c+fk9/s1OTmpsrIyHT58WA888ICGhoYcX7sk9fT06MyZMzp9+vRV1wrh5z8bxwUL8icQCOjjjz/W3//+93yXkpP7779fQ0NDisfjevvtt9Xe3q5QKJTvsuYkEolo+/bt6uvr05IlS/JdjjWO+1Xorrvu0m233XbVrncsFpPP58tTVfP3Vc1Ov5+Ojg698847+vDDDzOfhyN9Wf/U1JTGx8ezxjupfpfLpXvvvVdNTU0KBoNqbGzUK6+8UhC1h8NhjY2N6aGHHlJJSYlKSkoUCoW0f/9+lZSUyOv1Ov4eZuO4YHG5XGpqalJ/f3+mL51Oq7+/X36/P4+VzU9dXZ18Pl/W/SQSCZ06dcoR92OMUUdHhw4fPqxjx46prq4u63pTU5NKS0uz6h8eHtbly5cdUf9s0um0UqlUQdS+bt06nTt3TkNDQ5n28MMP6+mnn8782+n3MKt87x7Ppqenx7jdbvPGG2+YCxcumOeff95UVlaaaDSa79JmlUwmzdmzZ83Zs2eNJPPyyy+bs2fPmv/85z/GGGO6u7tNZWWl+etf/2o++ugjs3HjRlNXV2euXLmS58qN2bJli/F4POb48eNmdHQ007744ovMmM2bN5va2lpz7NgxMzg4aPx+v/H7/Xms+n927dplQqGQuXTpkvnoo4/Mrl27TFFRkfnggw+MMc6u/Vq+/q6QMYV5D44MFmOMefXVV01tba1xuVxm9erV5uTJk/ku6Zo+/PBDI+mq1t7eboz58i3nF154wXi9XuN2u826devM8PBwfov+f7PVLckcOnQoM+bKlStm69atZunSpeaOO+4wTzzxhBkdHc1f0V/z7LPPmu985zvG5XKZu+++26xbty4TKsY4u/Zr+WawFOI98HksAKxz3B4LgMJHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYN3/AUI6L9f4x+G0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49.0 320 [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, -5.0]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 2 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
